package lucene;

import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.*;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.analysis.*;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.queryParser.complexPhrase.ComplexPhraseQueryParser;
import org.apache.lucene.search.*;
import org.apache.lucene.search.highlight.InvalidTokenOffsetsException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.*;
 
import org.apache.lucene.search.vectorhighlight.FastVectorHighlighter;
import org.apache.lucene.search.vectorhighlight.FieldFragList;
import org.apache.lucene.search.vectorhighlight.FieldQuery;
import org.apache.lucene.search.vectorhighlight.FragListBuilder;
import org.apache.lucene.search.vectorhighlight.FragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.SimpleFragListBuilder;
import org.apache.lucene.search.vectorhighlight.SimpleFragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo.SubInfo;
import org.apache.lucene.search.vectorhighlight.FieldPhraseList.WeightedPhraseInfo.Toffs;
import org.apache.lucene.search.vectorhighlight.BaseFragmentsBuilder;
import org.apache.lucene.search.vectorhighlight.ScoreOrderFragmentsBuilder;

import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.NullFragmenter;
import org.apache.solr.analysis.HTMLStripCharFilter;

import java.io.*;
import java.util.*;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.IntRange;
import org.apache.commons.lang.math.Range;

import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import lucene.HighlighterUtil;

import php.java.servlet.RemoteHttpServletContextFactory;

/**
 * Servlet implementation class Lucene
 */
@WebServlet("/Searcher")
public class Searcher extends HttpServlet {

	private static final long serialVersionUID = 1L;
	// configuration
	// private static final String PRE_TAG = "<span style=\"background-color:#ffff00\">";
	// private static final String POST_TAG = "</span>";
	private static final String PRE_TAG = "<b>";
	private static final String POST_TAG = "</b>";
	
	private static final int CONTENT_FRAG_LEN = 250;
	private static final int TITLE_FRAG_LEN = -1;
	
    private Version currentVersion = Version.LUCENE_36;
	private Directory directory = null;
	private String indexPath = null;
	private String defaultPath = System.getProperty("user.dir") + "\\tomcat\\webapps\\lucene_space\\default_store";
	private IndexSearcher iSearcher = null;
	
    public Searcher() {
        super();
    }

	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		response.getWriter().write("do Get: Searcher - Lucene Servlet is running!\nThis servlet path: "+this.defaultPath);
	}

	protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		response.getWriter().write("do Post: Searcher - Lucene Servlet is running!");
	}

	protected void doPut(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
		
		RemoteHttpServletContextFactory ctx = new RemoteHttpServletContextFactory(this, getServletContext(), request, request, response);

		response.setHeader("X_JAVABRIDGE_CONTEXT", ctx.getId());
		response.setHeader("Pragma", "no-cache");
		response.setHeader("Cache-Control", "no-cache");

		try { 
		  ctx.handleRequests(request.getInputStream(), response.getOutputStream()); 
		} finally { 
		  ctx.destroy(); 
		}
	}
	
	public String hello() {return "Hello from MyServlet -  Searcher";}
		
	// @SuppressWarnings({ "rawtypes", "unchecked" })
	// public Hashtable search222(String path, String fieldName, String text) // test with 1 field only
		// throws IOException, ParseException, InvalidTokenOffsetsException {
		
		// IndexSearcher iSearcher = new IndexSearcher(IndexReader.open(FSDirectory.open(new File(path))));;
		// Analyzer analyzer = new StandardAnalyzer(this.currentVersion);
		// QueryParser parser = new ComplexPhraseQueryParser(this.currentVersion, fieldName, analyzer);
		// Query query = parser.parse(text);
		// TopDocs topDocs = iSearcher.search(query, 50);
	
	// }
	
	@SuppressWarnings({ "rawtypes", "unchecked" })
	public Hashtable search(String path, String fieldName, String text) // test with 1 field only
		throws IOException, ParseException, InvalidTokenOffsetsException {
	
		IndexSearcher iSearcher = new IndexSearcher(IndexReader.open(FSDirectory.open(new File(path))));;
		Analyzer analyzer = new StandardAnalyzer(this.currentVersion);
		QueryParser parser = new QueryParser(this.currentVersion, fieldName, analyzer);
		// QueryParser parser = new QueryParser(this.currentVersion, fieldName, new StandardSyntaxParser());
		Query query = parser.parse(text);
		// Query query = new TermQuery(term);
		// Term term = new Term(fieldName, "*"+text+"*");
		// TopDocs topDocs = iSearcher.search(query, 50000);
		TopDocs topDocs = iSearcher.search(query, 50);
		
		// re-format for php
		Hashtable result = new Hashtable();
		result.put("path", path);
		result.put("query", query.toString());
		result.put("totalHits", topDocs.totalHits);
		// append highlight
		//int fragLen = CONTENT_FRAG_LEN;
		//FastVectorHighlighter highlighter = getHighlighter(-1);
		
		FragListBuilder fragListBuilder = new SimpleFragListBuilder();
		// FragmentsBuilder fragBuilder = new ScoreOrderFragmentsBuilder( 
		FragmentsBuilder fragBuilder = new SimpleFragmentsBuilder( 
		                                       BaseFragmentsBuilder.COLORED_PRE_TAGS,     //beging of the mark: example; "<b style=\"background:yellow\">",....                   
		                                       BaseFragmentsBuilder.COLORED_POST_TAGS );   //end of the mark= "</b>"  
		FastVectorHighlighter highlighter = new FastVectorHighlighter(true, true, fragListBuilder, fragBuilder);
		
		FieldQuery fq = highlighter.getFieldQuery(query);


		// Get an array of references to matched documents
		ScoreDoc[] scoreDosArray = topDocs.scoreDocs;
		int i=0;
		ArrayList<Hashtable> allDocs = new ArrayList<Hashtable>(scoreDosArray.length);
		ArrayList<Hashtable> allDocsWithHighlight = new ArrayList<Hashtable>(scoreDosArray.length);
		for(ScoreDoc scoredoc: scoreDosArray){
			//Retrieve the matched document and show relevant details
			Document doc = iSearcher.doc(scoredoc.doc);
		  
			Hashtable docDetails = new Hashtable();
			docDetails.put(fieldName, doc.get(fieldName));
		  
			allDocs.add(docDetails);
			
			// String[] snippets = highlighter.getBestFragments(fq, iSearcher.getIndexReader(), scoredoc.doc, fieldName, 100/*, 10*/);
			String snippet = highlighter.getBestFragment(fq, iSearcher.getIndexReader(), scoredoc.doc, fieldName, 100/*, 10*/);
			// String snippet = "";
			// for (int j = 0; j < snippets.length; j++) {			 
			   // if (snippets[j] != null) snippet += (snippets[j] + "<br><br/>");			  
		   // }
			// String snippet = highlightHTML(analyzer, doc.get(fieldName), query, fieldName);
			Hashtable docDetailsWithHili = new Hashtable();
			docDetailsWithHili.put(fieldName, ""+snippet);
			allDocsWithHighlight.add(docDetailsWithHili);
		}
		result.put("docs", allDocs);
				
		result.put("hiliDocs", allDocsWithHighlight);		
		iSearcher.close();
		return result;
	}
	// ------------------------------------------- query entire HTML 1-----------------------------
	@SuppressWarnings("unchecked")
	public String highlightHTML(Analyzer analyzer, String htmlText, Query query, String fieldName) {

		QueryScorer scorer =  new QueryScorer(query, fieldName, fieldName);

		// SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter("\">", "");
		SimpleHTMLFormatter htmlFormatter = new SimpleHTMLFormatter();

		Highlighter highlighter = new Highlighter(htmlFormatter, scorer);
		
		// Nullfragmenter for highlighting entire document.
		highlighter.setTextFragmenter(new NullFragmenter());
		// TokenStream ts = analyzer.tokenStream(fieldName, new HTMLStripReader(new StringReader(htmlText)));
		TokenStream ts = analyzer.tokenStream(fieldName, new HTMLStripCharFilter(CharReader.get(new StringReader(htmlText)), Collections.EMPTY_SET, 65536));

		try {

			String highlightedText = highlighter.getBestFragment(ts, htmlText);

			if (highlightedText != null) {

				return highlightedText;

			}

		} catch (Exception e) {

			// LOG.error("Failed to highlight query string "+ query, e);
			return "Failed to highlight query string "+ query;

		}

		return htmlText;

	}
	
	
	
	//----------------------------------------------------------------------------------------------
	private FastVectorHighlighter getHighlighter(int fragLen) {
		FragListBuilder fragListBuilder = new SimpleFragListBuilder();
		// FragmentsBuilder fragBuilder = new MyFragmentsBuilder(PRE_TAG, POST_TAG, fragLen);
		FragmentsBuilder fragBuilder = new ScoreOrderFragmentsBuilder( 
		                                       BaseFragmentsBuilder.COLORED_PRE_TAGS,     //beging of the mark: example; "<b style=\"background:yellow\">",....                   
		                                       BaseFragmentsBuilder.COLORED_POST_TAGS );   //end of the mark= "</b>"  
		// FragmentsBuilder fragBuilder = new SimpleFragmentsBuilder(new String[] {PRE_TAG}, new String[] {POST_TAG});
		// FragmentsBuilder fragBuilder = new SimpleFragmentsBuilder();
		return new FastVectorHighlighter(true, true, fragListBuilder, fragBuilder);
		// return new FastVectorHighlighter(true, true);
	}
	
	private class MyFragmentsBuilder extends SimpleFragmentsBuilder {

		private int fragLen;
		private String pretag;
		private String posttag;
		
		public MyFragmentsBuilder(String pretag, String posttag, int fragLen) {
		  super(new String[] {pretag}, new String[] {posttag});
		  this.pretag = pretag;
		  this.posttag = posttag;
		  this.fragLen = fragLen;
		}
		
		@Override
		public String createFragment(IndexReader reader, int docId,
			String fieldName, FieldFragList fieldFragList ) 
			throws IOException {
			
		  // read the source string back from the index
		  Document doc = reader.document(docId);
		  String source = doc.get(fieldName);
		 
		  if (StringUtils.isEmpty(source)) {
			return source;
		  }
		  // find the occurrences of the matched phrase
		  List<Range> termPositions = new ArrayList<Range>();
		  List<WeightedFragInfo> fragInfos = new FieldFragList(this.fragLen).getFragInfos();
		  for (WeightedFragInfo fragInfo : fragInfos) {
			List<SubInfo> subInfos = fragInfo.getSubInfos();
			for (SubInfo subInfo : subInfos) {
			  List<Toffs> termOffsets = subInfo.getTermsOffsets();
			  for (Toffs termOffset : termOffsets) {
				Range termPosition = new IntRange(
				  termOffset.getStartOffset(), termOffset.getEndOffset());
				termPositions.add(termPosition);
			  }
			}
		  }
		  if (termPositions.size() == 0) {
			return StringUtils.substring(source, 0, fragLen);
		  }
		  int startFragPosition = 0;
		  int endFragPosition = 0;
		  // read back on the char array until we find a period,
		  // then read front until we find a letter/digit. This
		  // is our fragment start position. If no period found,
		  // then this must be the first sentence, start here.
		  if (fragLen < 0) {
			// we don't need a fragLen for titles, take them whole
			// so in this case fragLen should be -1.
			endFragPosition = source.length();
		  } else {
			int startTermPosition = termPositions.get(0).getMinimumInteger();
			char[] sourceChars = source.toCharArray();
			for (int i = startTermPosition; i >= 0; i--) {
			  if (sourceChars[i] == '.') {
				startFragPosition = i;
				break;
			  }
			}
			for (int i = startFragPosition; i < sourceChars.length; i++) {
			  if (Character.isLetterOrDigit(sourceChars[i])) {
				startFragPosition = i;
				break;
			  }
			}
			endFragPosition = 
			  Math.min(startFragPosition + fragLen, sourceChars.length);
		  }
		  // return the substring bounded by start and end, highlighting
		  // the matched phrase
		  final Range fragRange = 
			new IntRange(startFragPosition, endFragPosition);
		  CollectionUtils.filter(termPositions, new Predicate() {
			public boolean evaluate(Object obj) {
			  Range r = (Range) obj;
			  return (fragRange.containsRange(r));
			}
		  });
		  if (termPositions.size() == 0) {
			// unlikely, since we are pretty sure that there is at least
			// one term position in our fragRange, but just being paranoid
			return StringUtils.substring(source, startFragPosition, endFragPosition);
		  }
		  StringBuilder buf = new StringBuilder();
		  buf.append(StringUtils.substring(
			source, startFragPosition, 
			termPositions.get(0).getMinimumInteger()));
		  int numHighlights = termPositions.size();
		  for (int i = 0; i < numHighlights; i++) {
			buf.append(pretag).
			  append(StringUtils.substring(source, 
				termPositions.get(i).getMinimumInteger(), 
				termPositions.get(i).getMaximumInteger())).
			  append(posttag);
			if (i < numHighlights - 1) {
			  buf.append(StringUtils.substring(source, 
				termPositions.get(i).getMaximumInteger(), 
				termPositions.get(i+1).getMinimumInteger()));
			}
		  }
		  buf.append(StringUtils.substring(source, 
			termPositions.get(numHighlights-1).getMaximumInteger(), 
			fragRange.getMaximumInteger())); 
		  // return buf.toString();
		  return "hehehehe";
		}    
	  }
}
